# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

from scrapy.exceptions import DropItem
from scrapy.conf import settings
from pyley import CayleyClient, GraphObject

import codecs
import uuid


class DuplicatesPipeline(object):
    """
    对于爬取到的三元组进行去重判断，若已经在本次爬取过程中获取过，则忽略该三元组；
    在setting.py中设置的优先级为100，即先进行去重，再保存三元组。
    """

    def __init__(self):
        self.item_seen = set()

    def process_item(self, item, spider):
        triple_tuple = (item['subject'], item['tag'],
                        tuple(item['predicate']), tuple(item['object_value']), tuple(item['label']))
        if triple_tuple in self.item_seen:
            raise DropItem("Duplicate item found: %s" % item)
        else:
            self.item_seen.add(triple_tuple)
            return item


class BaikeAvpPipeline(object):
    """
    保存词条三元组的Pipeline，将每个页面的的三元组追加写入到baike_avp.nq；
    在setting.py中设置的优先级为200（值越大，优先级越低）。
    """

    def __init__(self):

        self.file = codecs.open('baidu_avp.nq', 'wb', 'utf-8')

        self.entity_namespace = uuid.UUID('e436436436436436436436436436436e')
        self.relation_namespace = uuid.UUID('a436436436436436436436436436436a')

        self.t_entity_name = '<%s> <name> "%s" <entity> .\n'
        self.t_entity_description = '<%s> <description> "%s" .\n'
        self.t_entity_type = '<%s> <type> <%s> .\n'
        self.t_entity_tag = '<%s> <tag> "%s" .\n'

        self.t_relation_name = '<%s> <name> "%s" <relation> .\n'
        self.t_attribute_name = '<%s> <name> "%s" <attribute> .\n'
        self.t_relation_confidence = '<%s> <conf> "%s" .\n'

        self.t_property_entity = '<%s> <%s> <%s> .\n'
        self.t_property_literal = '<%s> <%s> "%s" .\n'

    def process_item(self, item, spider):
        lines = []
        category_list = []
        relation_list = []
        avp_list = []
        object_entity_label_list = []
        entity_tag_list = []

        entity_repr = (item['subject'] + item['tag']).encode('utf-8')
        entity_id = str(uuid.uuid3(self.entity_namespace, entity_repr))

        entity_name = self.t_entity_name % (entity_id, item['subject'])
        entity_description = self.t_entity_description % (entity_id, item['desc'])
        entity_tag = self.t_entity_tag % (entity_id, item['tag'])
        entity_tag_list.append(entity_tag)

        for relation_item, object_item in zip(item['predicate'], item['object_value']):
            # relation_name = self.t_relation_name % (relation_id, relation_item)
            # relation_list.extend([relation_name])

            if object_item.startswith('|||'):
                object_item = object_item.lstrip('|||').split('|||')
                object_name = object_item[0]
                object_tag = object_item[1]
                object_repr = (object_name + object_tag).encode('utf-8')
                object_id = str(uuid.uuid3(self.entity_namespace, object_repr))
                relation_id = str(uuid.uuid3(self.relation_namespace, entity_id + relation_item.encode('utf-8') + object_id))
                relation_name = self.t_relation_name % (relation_id, relation_item)
                relation_conf = self.t_relation_confidence % (relation_id, '1.0')
                property_entity = self.t_property_entity % (entity_id, relation_id, object_id)
                object_entity_name = self.t_entity_name % (object_id, object_name)
                object_entity_tag = self.t_entity_tag % (object_id, object_tag)
                object_entity_label_list.append(object_entity_name)
                entity_tag_list.append(object_entity_tag)
                avp_list.extend([property_entity + relation_conf])
            else:
                relation_id = str(uuid.uuid3(self.relation_namespace, relation_item.encode('utf-8')))
                relation_name = self.t_attribute_name % (relation_id, relation_item)
                property_literal = self.t_property_literal % (entity_id, relation_id, object_item)
                avp_list.append(property_literal)
            relation_list.extend([relation_name])

        for label_item in item['label']:
            entity_type = self.t_entity_type % (entity_id, label_item)

            category_list.extend([entity_type])

        if object_entity_label_list:
            lines.extend([entity_name, entity_description] +
                         entity_tag_list + relation_list + category_list + avp_list + object_entity_label_list)
        else:
            lines.extend([entity_name, entity_description] +
                         entity_tag_list + relation_list + category_list + avp_list)

        self.file.writelines(lines)
        self.file.flush()
        return item


class CayleyPipeline(object):
    """
    保存词条三元组的Pipeline，将每个页面的的三元组追加写入到Cayley；
    在setting.py中设置的优先级为300（值越大，优先级越低）。
    """

    def __init__(self):

        self.client = CayleyClient('http://127.0.0.1:64210', 'v1')
        self.g = GraphObject()
        self.entity_namespace = uuid.UUID('e436436436436436436436436436436e')
        self.relation_namespace = uuid.UUID('a436436436436436436436436436436a')

        self.t_entity_name = '<%s> <name> "%s" <entity> .'
        self.t_entity_description = '<%s> <description> "%s" .'
        self.t_entity_type = '<%s> <type> <%s> .'
        self.t_entity_tag = '<%s> <tag> "%s" .'

        self.t_relation_name = '<%s> <name> "%s" <relation> .'
        self.t_attribute_name = '<%s> <name> "%s" <attribute> .'
        self.t_relation_confidence = '<%s> <conf> "%s" .'

        self.t_property_entity = '<%s> <%s> <%s> .'
        self.t_property_literal = '<%s> <%s> "%s" .'

    def process_item(self, item, spider):
        lines = []
        category_list = []
        relation_list = []
        avp_list = []
        object_entity_label_list = []
        entity_tag_list = []

        entity_repr = (item['subject'] + item['tag']).encode('utf-8')
        entity_id = str(uuid.uuid3(self.entity_namespace, entity_repr))

        entity_name = self.t_entity_name % (entity_id, item['subject'])
        entity_description = self.t_entity_description % (entity_id, item['desc'])
        entity_tag = self.t_entity_tag % (entity_id, item['tag'])
        entity_tag_list.append(entity_tag)

        for relation_item, object_item in zip(item['predicate'], item['object_value']):
            # relation_name = self.t_relation_name % (relation_id, relation_item)
            # relation_list.extend([relation_name])

            if object_item.startswith('|||'):
                object_item = object_item.lstrip('|||').split('|||')
                object_name = object_item[0]
                object_tag = object_item[1]
                object_repr = (object_name + object_tag).encode('utf-8')
                object_id = str(uuid.uuid3(self.entity_namespace, object_repr))
                relation_id = str(
                    uuid.uuid3(self.relation_namespace, entity_id + relation_item.encode('utf-8') + object_id))
                relation_name = self.t_relation_name % (relation_id, relation_item)
                relation_conf = self.t_relation_confidence % (relation_id, '1.0')
                property_entity = self.t_property_entity % (entity_id, relation_id, object_id)
                object_entity_name = self.t_entity_name % (object_id, object_name)
                object_entity_tag = self.t_entity_tag % (object_id, object_tag)
                object_entity_label_list.append(object_entity_name)
                entity_tag_list.append(object_entity_tag)
                avp_list.extend([property_entity, relation_conf])
            else:
                relation_id = str(uuid.uuid3(self.relation_namespace, relation_item.encode('utf-8')))
                relation_name = self.t_attribute_name % (relation_id, relation_item)
                property_literal = self.t_property_literal % (entity_id, relation_id, object_item)
                avp_list.append(property_literal)
            relation_list.extend([relation_name])

        for label_item in item['label']:
            entity_type = self.t_entity_type % (entity_id, label_item)

            category_list.extend([entity_type])

        if object_entity_label_list:
            lines.extend([entity_name, entity_description] +
                         entity_tag_list + relation_list + category_list + avp_list + object_entity_label_list)
        else:
            lines.extend([entity_name, entity_description] +
                         entity_tag_list + relation_list + category_list + avp_list)

        quads = [line.split() for line in lines]
        self.client.AddQuads(quads)
        return item
